package com.example.myFirstWebmagic.company;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.Map.Entry;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import com.example.myFirstWebmagic.company.entity.Company;
import com.example.myFirstWebmagic.company.entity.CompanyJcfa;
import com.example.myFirstWebmagic.company.entity.CompanyJcsj;
import com.example.myFirstWebmagic.company.entity.CompanyJcsjPK;
import com.example.myFirstWebmagic.company.entity.CompanyJcxm;
import com.example.myFirstWebmagic.company.entity.CompanyJcxmPK;
import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import com.example.myFirstWebmagic.pipeline.CompanyInfoPipeline;
import com.example.myFirstWebmagic.util.CastUtil;
import com.example.myFirstWebmagic.util.StringUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

/**
 * 
 * ClassName: CompanyProcess 
 * @Description: 企业信息监测
 * @author xiongyao
 * @date 2017年3月15日
 */
@Component("companyProcess")
public class CompanyProcess implements PageProcessor{

	
	private static final Logger LOGGER=LoggerFactory.getLogger(CompanyProcess.class);
	
	private Site site = Site.me().setRetryTimes(10).setSleepTime(10000);
	
	//翻页查询
	private static final String QUERY_URL="http://59.172.182.106/qyjc/jcjgXxOne.rh?pageNumber=";
	
	List<String> targetReuqest=new ArrayList<String>();
	
	//环保厅的首页
	private static final String HOME_URL="http://59.172.182.106/qyjc/jcjgXxOne.rh";
	
	List<String> companyUrlList=new ArrayList<String>();
	
	//查询企业监测首页
	private static final String COMPANY_URL="http://59.172.182.106/qyjc/jcjgXxTwo.rh?id=";
	
	@Override
	public void process(Page page) {
		// TODO 如果是在平台主页获取企业主页链接地址
		Html html = page.getHtml();
		if(page.getRequest().getUrl().equals(HOME_URL)||page.getRequest().getUrl().startsWith(QUERY_URL)){
			String pageNumString=html.xpath("//*[@id='queryForm']/table[2]/tbody/tr/td/table[1]/tbody/tr/td/table[3]/tbody/tr[2]/td/table/tbody/tr[1]/td[2]/table[3]/tbody/tr/td/div/table/tbody/tr/td/text()").get();
			int pageNum;
			if(StringUtil.isNotEmpty(pageNumString)){
				pageNum=CastUtil.castInt(pageNumString.substring(pageNumString.lastIndexOf("共")+1, pageNumString.indexOf("页")).trim()) ;
				for(int i=2;i<=pageNum;i++){
					targetReuqest.add(QUERY_URL+i);
				}
			}
			companyUrlList=html.xpath("*[@id='queryForm']/table[2]/tbody/tr/td/table[1]/tbody/tr/td/table[3]/tbody/tr[2]/td/table/tbody/tr[1]/td[2]/table[2]/tbody/tr[1]/td[1]/table[1]/tbody//tr/td[1]/div/a/@href").all();
			List<String> targetUrl=new ArrayList<String>();
			targetUrl.addAll(targetReuqest);
			targetUrl.addAll(companyUrlList);
			page.addTargetRequests(targetUrl);
		}
		// TODO 如果是企业主页
		if (page.getRequest().getUrl().startsWith(COMPANY_URL)) {
			// TODO 获取企业基础信息，自行监测方案和最新监测数据
			Company company = getCompanyMainInfo(html);
			// 截取企业ID
			String url=page.getRequest().getUrl().toString();
			String id=url.substring(url.indexOf("=")+1,url.length());
			company.setId(id);
			
			//设置企业监测方案ID
			CompanyJcfa companyJcfa=company.getCompanyJcfa();
			companyJcfa.setId(id);
			company.setCompanyJcfa(companyJcfa);
			
			
			//设置企业监测方案中监测项目的ID
			Set<CompanyJcxm> jcxms= companyJcfa.getCompanyJcxms();
			for(CompanyJcxm jcxm:jcxms){
				CompanyJcxmPK jcxmPK=jcxm.getId();
				jcxmPK.setId(id);
				jcxm.setId(jcxmPK);
			}
			companyJcfa.setCompanyJcxms(jcxms);
			
			page.putField("company", company);
			
			LOGGER.info(company.toString());
		}
		
		
	}

	/**
	 * 
	 * @Description: 获取企业的主要信息
	 * @param @param html
	 * @param @return   
	 * @return EnCompanyInfo  
	 * @throws
	 * @author xiongyao
	 * @date 2017年3月20日
	 */
	private Company getCompanyMainInfo(Html html) {
		// TODO 获取企业基础信息
		Company company=getJcinfo(html);
		// TODO 获取企业自行监测方案
		company.setCompanyJcfa(getZcjcfa(html));
		// TODO 获取企业最新监测数据
		company.setCompanyJcsjs(getNewestJcsj(html));
		return company;
	}
	
	/**
	 * 
	 * @Description: 获取最新监测数据
	 * @param @param html
	 * @param @return   
	 * @return Map<String,Queue<Map<String,String>>>  
	 * @throws
	 * @author xiongyao
	 * @date 2017年3月23日
	 */
	private Set<CompanyJcsj> getNewestJcsj(Html html) {
		//最新监测数据
		Map<String, Queue<Map<String, String>>> zxjcData=new LinkedHashMap<String, Queue<Map<String,String>>>();
		
		
		
		
		String zdjc=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[1]/td/div/span/text()").get();
		//列名//*[@id="jcjg"]/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[2]
		List<String> columnNames=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[2]//span/text()").all();
		
		Queue<Map<String, String>> zdkcQueue=new LinkedList<Map<String,String>>();
		//监测点位
		List<String> jcdwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[1]/text()").all();
		for (String jcdw : jcdwlist) {
			//一行自动监测
			Map<String, String> zdjcxm=new LinkedHashMap<String, String>();
			zdjcxm.put(columnNames.get(0), jcdw);
			zdkcQueue.offer(zdjcxm);
		}
		
		
		
		//监测时间
		List<String> zcsjlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[2]/text()").all();
		for (String zcsj : zcsjlist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(1), zcsj);
			zdkcQueue.offer(zdjcxm);
		}
		//监测项目
		List<String> zcxmlist=null;
		//如果是手动监测
		if(zdjc.equals("手动监测")){
			zcxmlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[3]/text()").all();
			for (String zcxm : zcxmlist) {
				Map<String, String> zdjcxm=zdkcQueue.poll();
				zdjcxm.put(columnNames.get(2), zcxm);
				zdkcQueue.offer(zdjcxm);
			}
		}else{
			//如果是自动监测
			zcxmlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[3]/p[1]/text()").all();
			for (String zcxm : zcxmlist) {
				Map<String, String> zdjcxm=zdkcQueue.poll();
				zdjcxm.put(columnNames.get(2), zcxm);
				zdkcQueue.offer(zdjcxm);
			}
		}
		//污染物浓度
		List<String> wrwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[4]/text()").all();
		for (String wrw : wrwlist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(3), wrw);
			zdkcQueue.offer(zdjcxm);
		}
		//排放限值
		List<String> pfxzlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[5]/text()").all();
		for (String pfxz : pfxzlist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(4), pfxz);
			zdkcQueue.offer(zdjcxm);
		}
		//单位
		List<String> dwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[6]/text()").all();
		for (String dw : dwlist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(5), dw);
			zdkcQueue.offer(zdjcxm);
		}
		//是否达标
		List<String> sfdblist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[7]/img/@src").all();
		for (String sfdb : sfdblist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(6), sfdb);
			zdkcQueue.offer(zdjcxm);
		}
		//超标倍数
		List<String> cbbslist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[8]/text()").all();
		for (String cbbs : cbbslist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(7),cbbs);
			zdkcQueue.offer(zdjcxm);
		}
		//备注
		List<String> bzlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[@id]/td[9]/text()").all();
		for (String bz : bzlist) {
			Map<String, String> zdjcxm=zdkcQueue.poll();
			zdjcxm.put(columnNames.get(8), bz);
			zdkcQueue.offer(zdjcxm);
		}
		
		zxjcData.put(zdjc, zdkcQueue);
		
		
		//手动监测
		//*[@id="jcjg"]/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[1]/table/tbody/tr[1]/td/div/span
		//*[@id="jcjg"]/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[1]/td/div/span
		                        //*[@id="jcjg"]/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div/table/tbody/tr[1]/td/div/span
		String sdjc=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[1]/td/div/span/text()").get();
		
		Queue<Map<String, String>> sdkcQueue=new LinkedList<Map<String,String>>();
		//监测点位
		jcdwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[1]/text()").all();
		for (String jcdw : jcdwlist) {
			//一行自动监测
			Map<String, String> zdjcxm=new LinkedHashMap<String, String>();
			zdjcxm.put(columnNames.get(0), jcdw);
			sdkcQueue.offer(zdjcxm);
		}
		
		
		
		//监测时间
		zcsjlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[2]/text()").all();
		for (String zcsj : zcsjlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(1), zcsj);
			sdkcQueue.offer(zdjcxm);
		}
		//监测项目
		zcxmlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[3]/text()").all();
		for (String zcxm : zcxmlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(2), zcxm);
			sdkcQueue.offer(zdjcxm);
		}
		//污染物浓度
		wrwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[4]/text()").all();
		for (String wrw : wrwlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(3), wrw);
			sdkcQueue.offer(zdjcxm);
		}
		//排放限值
		pfxzlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[5]/text()").all();
		for (String pfxz : pfxzlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(4), pfxz);
			sdkcQueue.offer(zdjcxm);
		}
		//单位
		dwlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[6]/text()").all();
		for (String dw : dwlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(5), dw);
			sdkcQueue.offer(zdjcxm);
		}
		//是否达标
		sfdblist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[7]/img/@src").all();
		for (String sfdb : sfdblist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(6), sfdb);
			sdkcQueue.offer(zdjcxm);
		}
		//超标倍数
		cbbslist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[8]/text()").all();
		for (String cbbs : cbbslist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(7),cbbs);
			sdkcQueue.offer(zdjcxm);
		}
		//备注
		bzlist=html.xpath("//*[@id='jcjg']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td/div[2]/table/tbody/tr[@id]/td[9]/text()").all();
		for (String bz : bzlist) {
			Map<String, String> zdjcxm=sdkcQueue.poll();
			zdjcxm.put(columnNames.get(8), bz);
			sdkcQueue.offer(zdjcxm);
		}
		
		zxjcData.put(sdjc, sdkcQueue);
		
		//LOGGER.info(zxjcData.toString());
		
		//监测数据
		Set<CompanyJcsj> companyJcsjs=new HashSet<CompanyJcsj>();
		//获取自动监测数据
		Queue<Map<String, String>> zdjcSjs=zxjcData.get("自动监测");
		if(CollectionUtils.isNotEmpty(zdjcSjs)){
			for(Map<String, String> map:zdjcSjs){
				CompanyJcsj jcsj=new CompanyJcsj();
				CompanyJcsjPK jcsjPK=new CompanyJcsjPK();
				jcsjPK.setMonitoringTime(map.get("监测时间"));
				jcsjPK.setMonitoringProject(map.get("监测项目"));
				jcsjPK.setMonitoringPoint(map.get("监测点位"));
				jcsj.setId(jcsjPK);
				jcsj.setPollutantConcentration(map.get("污染物浓度"));
				jcsj.setEmissionLimit(map.get("排放限值"));
				jcsj.setUnit(map.get("单位"));
				jcsj.setWhetherCompliance(map.get("是否达标"));
				jcsj.setRemarks(map.get("备注"));
				jcsj.setMonitoringType("自动监测");		
				companyJcsjs.add(jcsj);
			}
		}
		
		
		//获取手动监测数据
		Queue<Map<String, String>> sdjcSjs=zxjcData.get("手动监测");
		if(CollectionUtils.isNotEmpty(sdjcSjs)){
		
			for(Map<String, String> map:sdjcSjs){
				CompanyJcsj jcsj=new CompanyJcsj();
				CompanyJcsjPK jcsjPK=new CompanyJcsjPK();
				jcsjPK.setMonitoringTime(map.get("监测时间"));
				jcsjPK.setMonitoringProject(map.get("监测项目"));
				jcsjPK.setMonitoringPoint(map.get("监测点位"));
				jcsj.setId(jcsjPK);
				jcsj.setPollutantConcentration(map.get("污染物浓度"));
				jcsj.setEmissionLimit(map.get("排放限值"));
				jcsj.setUnit(map.get("单位"));
				jcsj.setWhetherCompliance(map.get("是否达标"));
				jcsj.setRemarks(map.get("备注"));
				jcsj.setMonitoringType("手动监测");		
				companyJcsjs.add(jcsj);	
			}
		}
		
		return companyJcsjs;
	}
	

	/**
	 * 
	 * @Description: 获取企业基础信息
	 * @param @param html
	 * @param @return   
	 * @return EnCompanyInfo  
	 * @throws
	 * @author xiongyao
	 * @date 2017年3月23日
	 */
	private Company getJcinfo(Html html) {
		Company company=new Company();
		//===========================================获取企业基础信息===========================================================
		//企业介绍
		String companyInfo=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[1]/tbody/tr[1]/td[3]/table/tbody/tr/td/table/tbody/tr/td/textarea/text()").get();
		company.setInfo(companyInfo);
		//企业名称
		String companyName=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[1]/td[2]/span/text()").get();
		company.setName(companyName);
		//法人代表//
		String corporateRepresentative=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[1]/td[4]/span/text()").get();
		company.setLegalPerson(corporateRepresentative);
		//详细地址
		String address=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[2]/td[2]/span/text()").get();
		company.setAddress(address);
		//组织机构代码
		String companyCode=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[2]/td[4]/span/text()").get();
		company.setCode(companyCode);
		//企业网址
		String companyWebsite=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[3]/td[2]/span/a/text()").get();
		company.setWebsite(companyWebsite);
		//所属行业
		String industry=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[3]/td[4]/span/text()").get();
		company.setIndustry(industry);
		//污染源类型
		String pollutantSourceType=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[4]/td[2]/span/text()").get();
		company.setPollutantSourceType(pollutantSourceType);
		//生产周期
		String productionCycle=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[4]/td[4]/span/text()").get();
		company.setProductionCycle(productionCycle);
		//联系人
		String contactMan=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[5]/td[2]/span/text()").get();
		company.setContactMan(contactMan);
		//联系电话
		String contactPhone=html.xpath("//*[@id='link_info1']/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table/tbody/tr/td/table[2]/tbody/tr[5]/td[4]/span/text()").get();
		company.setContactPhone(contactPhone);
		
		return company;
	}

	
	
	/**
	 * 
	 * @Description: 获取自行监测方案
	 * @param @param html
	 * @param @return   
	 * @return Map<String,Object>  
	 * @throws
	 * @author xiongyao
	 * @date 2017年3月23日
	 */
	private CompanyJcfa getZcjcfa(Html html) {
		//企业监测方案
		CompanyJcfa companyJcfa=new CompanyJcfa();
		//监测项目
		Set<CompanyJcxm> companyJcxms=new HashSet<CompanyJcxm>();
		
		
		Map<String, Object> selfMonitoringScheme=new LinkedHashMap<String, Object>(); 
		//行政区
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[2]/text()").get());
		companyJcfa.setDistrict(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[2]/text()").get());
		//行业
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[3]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[4]/text()").get());
		companyJcfa.setIndustry(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[2]/td[4]/text()").get());
		//企业类型
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[2]/text()").get());
		companyJcfa.setType(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[2]/text()").get());
		//污染物种类
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[3]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[4]/text()").get());
		companyJcfa.setPollutantSourceType(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[3]/td[4]/text()").get());
		//排口属性
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[1]/div/text()").get(),html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[2]/text()").get());
		companyJcfa.setExitAttribute(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[2]/text()").get());
		//废水是否外排
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[3]/div/text()").get(),html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[4]/text()").get());
		companyJcfa.setIfDrainOut(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[4]/td[4]/text()").get());
		//产污工艺简述
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[5]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[5]/td[2]/text()").get());
		companyJcfa.setBriefIntroductionOfSewageProductionProcess(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[5]/td[2]/text()").get());
		//污染处理工艺简述
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[6]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[6]/td[2]/text()").get());
		companyJcfa.setBriefIntroductionOfPollutionTreatmentTechnology(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[6]/td[2]/text()").get());
		//委托机构名称
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[7]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[7]/td[2]/text()").get());
		companyJcfa.setNameOfAgency(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[7]/td[2]/text()").get());
		//执行排放标准
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[8]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[8]/td[2]/text()").get());
		companyJcfa.setDischargeStandards(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[8]/td[2]/text()").get());
		//排放去向
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[9]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[9]/td[2]/text()").get());
		companyJcfa.setEmissionFate(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[9]/td[2]/text()").get());
		//质量控制措施
		selfMonitoringScheme.put(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[10]/td[1]/div/text()").get(), html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[10]/td[2]/text()").get());
		companyJcfa.setQualityControlMeasurement(html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody/tr[10]/td[2]/text()").get());
		
		//监测点位名称
		List<String> jcdwNames=html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody//td[@rowspan]/text()").all();
		
		
		//监测点位对应的rowspan
		List<String> jcdwRowspans=html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody//td[@rowspan]/@rowspan").all();
		
		List<String> contentList= html.xpath("//*[@id='jcfa']/tbody/tr/td/table/tbody/tr/td/table/tbody//*/text()").all();
		

		List<String> jdxmColumn=new ArrayList<String>();
		
		for(int i=contentList.indexOf("监测点位");i<contentList.indexOf("监测点位")+9;i++){
			jdxmColumn.add(contentList.get(i));
		}
		
		
		Map<String, List<Map<String, String>>> map=new HashMap<String, List<Map<String,String>>>();
		for(String jcdwName:jcdwNames){
			//监测点位有的监测项目个数
			int rowNums=CastUtil.castInt(jcdwRowspans.get(jcdwNames.indexOf(jcdwName)));
			//监测点位的监测项目列表
			List<Map<String, String>> jcxmList=new ArrayList<Map<String,String>>();
			for(int i=0;i<rowNums;i++){
				//一个监测项目
				Map<String, String> jcxmMap=new LinkedHashMap<String, String>();
				jcxmList.add(jcxmMap);
			}
			int index=0;
			for(int j=contentList.indexOf(jcdwName);j<contentList.indexOf(jcdwName)+9*CastUtil.castInt(jcdwRowspans.get(jcdwNames.indexOf(jcdwName)));j++){
				//index/9=行号
				int rowNum=index/9;
				int columnNum=index%9;
				//内容
				String content=contentList.get(j);
				//对应列名
				String columnName=jdxmColumn.get(columnNum);
				jcxmList.get(rowNum).put(columnName, content);
				jcxmList.get(rowNum).put("监测点位", jcdwName);
				index++;
			}
			map.put(jcdwName, jcxmList);
		}
		
		for(Entry<String, List<Map<String, String>>> entry: map.entrySet()){
			List<Map<String, String>> list=entry.getValue();
			for(Map<String, String> jcxmMap:list){
				CompanyJcxm jcxm=new CompanyJcxm();
				CompanyJcxmPK jcxmPK=new CompanyJcxmPK();
				jcxmPK.setMonitoringPoint(jcxmMap.get("监测点位"));
				jcxmPK.setMonitoringProject(jcxmMap.get("监测项目"));
				jcxm.setId(jcxmPK);
				jcxm.setEmissionLimit(jcxmMap.get("排放限值"));
				jcxm.setMonitoringMethod(jcxmMap.get("监测方法"));
				jcxm.setAnalysisInstrument(jcxmMap.get("分析仪器"));
				jcxm.setMonitoringType(jcxmMap.get("监测方式"));
				jcxm.setMonitoringMeans(jcxmMap.get("监测手段"));
				jcxm.setMonitoringFrequency(jcxmMap.get("监测频次"));
				jcxm.setReleaseTime(jcxmMap.get("发布时限"));
				companyJcxms.add(jcxm);
			}
		}
		
		selfMonitoringScheme.put("监测列表", map);
		
		companyJcfa.setCompanyJcxms(companyJcxms);
		
		return companyJcfa;
	}
	
	
	

	@Test
	public void testProcess(){
		String [] startUrl=new String[]{"http://59.172.182.106/qyjc/jcjgXxTwo.rh?id=2887"};
		// TODO Auto-generated method stub
		Spider.create(new CompanyProcess())
        // TODO 从"http://datacenter.mep.gov.cn/index!getCityMain.action"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo())
        // TODO 开启5个线程抓取
        .thread(5)
        // TODO 启动爬虫
        .run();
	}
	
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

}
